# -*- coding: utf-8 -*-
import json
from waterwalk.items import *
from waterwalk.utils import *
import datetime
import urllib
import requests
from waterwalk.settings import USER_AGENTS,PROXY_LIST,CRAWL_START,CRAWL_END,CITYS
import random


KEY = 'd306zoyjsyarp7ifhu67rjxn52tv0t20'
class AbownersSpider(scrapy.Spider):
    name = 'abowners'
    allowed_domains = ['airbnbchina.cn']
    custom_settings = {
        'DOWNLOAD_DELAY': 0.28,
    }
    list_url = "https://www.airbnbchina.cn/api/v2/user_promo_listings?currency=CNY&locale=zh&"
    detail_url = "https://www.airbnbchina.cn/rooms/"

    def start_requests(self):
        for city in ['xian', 'chengdu']:
            for owner in get_owners(city):
                params = {
                    "user_id": owner['_id'],
                    "_limit": 200,
                    "_offset": 0,
                    "key": KEY
                }
                url = self.list_url + urllib.parse.urlencode(params)
                request = scrapy.Request(url=url, callback=self.parse)
                request.meta['owner_id'] = owner['_id']
                yield request

    def parse(self, response):
        '''
        解析列表分页
        '''
        body_dict = json.loads(response.body.decode('utf-8'))
        for h in body_dict['user_promo_listings']:
            h_url = self.detail_url + str(h['id'])
            request = scrapy.Request(h_url, self.parse_page)
            house = HouseItem()
            house['city'] = h['localized_city']
            house['price'] = h['nightly_price_as_guest']
            house['room_num'] = h['bedrooms']
            house['bed_num'] = h['beds']
            house['bathroom_num'] = h['bathrooms']
            house['owner_id'] = response.meta['owner_id']
            # 独立房间/整套
            if h['space_type'].find('独立') != -1:
                house['room_type'] = 1
            else:
                house['room_type'] = 2
            request.meta['house'] = house
            yield request

    def parse_page(self, response):
        """
        普通页面规则
        :param response:
        :param house:
        :return:
        """
        house = response.meta['house']
        house['collection'] = get_collection_name(house['city'])
        house['title'] = response.css('h1._tpbrp::text').extract_first()
        house['content'] = trim(''.join(response.css('p._6z3til span span::text').extract()))

        #评价
        try:
            house['review_num'] = int(response.css('span._1cy09umr span::text').re_first(r'(\d+)'))
            house['rating'] = int(response.css('div._16tcko6 span::attr(aria-label)').re_first(r'(\d)'))
        except Exception:
            house['review_num'] = 0
            house['rating'] = 0

        house['pictures'] = response.css('div._3nukz4e img::attr(src)').extract()

        # longti,lati
        house['longi'] = response.css('body').re_first(r'lng":(\d+\.\d+)')
        house['lati'] = response.css('body').re_first(r'lat":(\d+\.\d+)')
        house['position'] = {
            'type': 'Point',
            'coordinates': [float(house['longi']), float(house['lati'])]
        }

        house['uniqe_key'] = int(response.url.split(r'/')[-1])
        # date and unique_key
        house['crawl_date'] = datetime.datetime.now()
        # collection_name

        house['source_url'] = response.url

        # 日历
        cl_url = 'https://www.airbnbchina.cn/api/v2/calendar_months?_format=with_conditions&count=4&currency=CNY&locale=zh&'
        params = {
            "listing_id": house['uniqe_key'],
            "key": KEY
        }
        url = cl_url + urllib.parse.urlencode(params)
        request = scrapy.Request(url, self.calendar)
        request.meta['house'] = house
        yield request

    def calendar(self, response):
        house = response.meta['house']
        body_dict = json.loads(response.body.decode('utf-8'))
        house['calendar'] = [{'month': m['month'], 'days': m['days']} for m in body_dict['calendar_months']]
        # 评价
        rv_url = 'https://www.airbnbchina.cn/api/v2/reviews?currency=CNY&locale=zh&role=guest&_format=for_p3&_limit=100&_offset=0&_order=language_country&'
        params = {
            "listing_id": house['uniqe_key'],
            "key": KEY
        }
        url = rv_url + urllib.parse.urlencode(params)
        request = scrapy.Request(url,self.review)
        request.meta['house'] = house
        yield request

    def review(self, response):
        house = response.meta['house']
        body_dict = json.loads(response.body.decode('utf-8'))
        house['reviews'] = [{'comments': r['comments'], 'date': r['created_at'], 'rating': r['rating']} for r in body_dict['reviews']]
        # 房东
        owner_url = 'https://www.airbnbchina.cn/users/show/' + str(house['owner_id'])
        url = owner_url
        request = scrapy.Request(url, self.owner)
        request.meta['house'] = house
        yield request

    def owner(self, response):
        house = response.meta['house']
        try:
            house['owner_review_num'] = int(response.css('#reviews h2 small::text').re_first(r'(\d+)'))
            house['owner_house_num'] = int(response.css('h2.space-3 small::text').re_first(r'(\d+)'))
            house['owner_des'] = response.css('div.space-top-2 p::text').extract_first()
            house['owner_reg_date'] = trim(response.css('div.space-top-2 span.text-normal::text').extract_first())
        except Exception:
            pass
        return house